

<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<head>
  <meta charset="utf-8" />
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
  
  <title>mindspore.nn.layer.normalization &mdash; MindSpore master documentation</title>
  

  
  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />

  
  

  
  

  

  
  <!--[if lt IE 9]>
    <script src="../../../../_static/js/html5shiv.min.js"></script>
  <![endif]-->
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
        <script src="../../../../_static/jquery.js"></script>
        <script src="../../../../_static/underscore.js"></script>
        <script src="../../../../_static/doctools.js"></script>
        <script src="../../../../_static/language_data.js"></script>
        <script async="async" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
    
    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>

    
    <link rel="index" title="Index" href="../../../../genindex.html" />
    <link rel="search" title="Search" href="../../../../search.html" /> 
</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

          
            <a href="../../../../index.html" class="icon icon-home"> MindSpore
          

          
          </a>

          
            
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        
        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption"><span class="caption-text">MindSpore Python API</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.html">mindspore</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.common.initializer.html">mindspore.common.initializer</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.communication.html">mindspore.communication</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.compression.html">mindspore.compression</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.context.html">mindspore.context</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.html">mindspore.dataset</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.audio.html">mindspore.dataset.audio</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.config.html">mindspore.dataset.config</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.text.html">mindspore.dataset.text</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.transforms.html">mindspore.dataset.transforms</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.vision.html">mindspore.dataset.vision</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.mindrecord.html">mindspore.mindrecord</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.nn.html">mindspore.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.nn.probability.html">mindspore.nn.probability</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.nn.transformer.html">mindspore.nn.transformer</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.numpy.html">mindspore.numpy</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.ops.html">mindspore.ops</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.parallel.html">mindspore.parallel</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.parallel.nn.html">mindspore.parallel.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.profiler.html">mindspore.profiler</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.scipy.html">mindspore.scipy</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.train.html">mindspore.train</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.boost.html">mindspore.boost</a></li>
</ul>
<p class="caption"><span class="caption-text">MindSpore C++ API</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://www.mindspore.cn/lite/api/zh-CN/master/api_cpp/mindspore.html">MindSpore Lite↗</a></li>
</ul>

            
          
        </div>
        
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../../../../index.html">MindSpore</a>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        
          

















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../../../../index.html" class="icon icon-home"></a> &raquo;</li>
        
          <li><a href="../../../index.html">Module code</a> &raquo;</li>
        
      <li>mindspore.nn.layer.normalization</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <h1>Source code for mindspore.nn.layer.normalization</h1><div class="highlight"><pre>
<span></span><span class="c1"># Copyright 2020-2021 Huawei Technologies Co., Ltd</span>
<span class="c1">#</span>
<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
<span class="c1"># you may not use this file except in compliance with the License.</span>
<span class="c1"># You may obtain a copy of the License at</span>
<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
<span class="c1"># See the License for the specific language governing permissions and</span>
<span class="c1"># limitations under the License.</span>
<span class="c1"># ============================================================================</span>
<span class="sd">&quot;&quot;&quot;normalization&quot;&quot;&quot;</span>
<span class="kn">import</span> <span class="nn">itertools</span>
<span class="kn">import</span> <span class="nn">numbers</span>

<span class="kn">from</span> <span class="nn">mindspore.ops</span> <span class="kn">import</span> <span class="n">operations</span> <span class="k">as</span> <span class="n">P</span>
<span class="kn">from</span> <span class="nn">mindspore.ops</span> <span class="kn">import</span> <span class="n">functional</span> <span class="k">as</span> <span class="n">F</span>
<span class="kn">from</span> <span class="nn">mindspore.ops.operations</span> <span class="kn">import</span> <span class="n">_inner_ops</span> <span class="k">as</span> <span class="n">inner</span>
<span class="kn">from</span> <span class="nn">mindspore.common.parameter</span> <span class="kn">import</span> <span class="n">Parameter</span>
<span class="kn">from</span> <span class="nn">mindspore.common.initializer</span> <span class="kn">import</span> <span class="n">initializer</span><span class="p">,</span> <span class="n">Initializer</span>
<span class="kn">from</span> <span class="nn">mindspore.common.tensor</span> <span class="kn">import</span> <span class="n">Tensor</span>
<span class="kn">from</span> <span class="nn">mindspore.common._decorator</span> <span class="kn">import</span> <span class="n">deprecated</span>
<span class="kn">from</span> <span class="nn">mindspore.ops.primitive</span> <span class="kn">import</span> <span class="n">constexpr</span>
<span class="kn">import</span> <span class="nn">mindspore.context</span> <span class="k">as</span> <span class="nn">context</span>
<span class="kn">from</span> <span class="nn">mindspore._checkparam</span> <span class="kn">import</span> <span class="n">Rel</span>
<span class="kn">from</span> <span class="nn">mindspore._checkparam</span> <span class="kn">import</span> <span class="n">Validator</span> <span class="k">as</span> <span class="n">validator</span>
<span class="kn">from</span> <span class="nn">mindspore._extends</span> <span class="kn">import</span> <span class="n">cell_attr_register</span>
<span class="kn">from</span> <span class="nn">mindspore.communication.management</span> <span class="kn">import</span> <span class="n">get_group_size</span><span class="p">,</span> <span class="n">get_rank</span>
<span class="kn">from</span> <span class="nn">mindspore.communication</span> <span class="kn">import</span> <span class="n">management</span>
<span class="kn">from</span> <span class="nn">mindspore.common</span> <span class="kn">import</span> <span class="n">dtype</span> <span class="k">as</span> <span class="n">mstype</span>
<span class="kn">from</span> <span class="nn">mindspore.parallel._utils</span> <span class="kn">import</span> <span class="n">_is_in_auto_parallel_mode</span>
<span class="kn">from</span> <span class="nn">..cell</span> <span class="kn">import</span> <span class="n">Cell</span>

<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;BatchNorm1d&#39;</span><span class="p">,</span> <span class="s1">&#39;BatchNorm2d&#39;</span><span class="p">,</span> <span class="s1">&#39;BatchNorm3d&#39;</span><span class="p">,</span> <span class="s1">&#39;LayerNorm&#39;</span><span class="p">,</span> <span class="s1">&#39;GroupNorm&#39;</span><span class="p">,</span>
           <span class="s1">&#39;GlobalBatchNorm&#39;</span><span class="p">,</span> <span class="s1">&#39;SyncBatchNorm&#39;</span><span class="p">,</span> <span class="s1">&#39;InstanceNorm2d&#39;</span><span class="p">]</span>

<span class="n">SYNC_BN_GROUP_NAME</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>


<span class="k">class</span> <span class="nc">_BatchNorm</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Batch Normalization base class.&quot;&quot;&quot;</span>

    <span class="nd">@cell_attr_register</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
                 <span class="n">num_features</span><span class="p">,</span>
                 <span class="n">eps</span><span class="o">=</span><span class="mf">1e-5</span><span class="p">,</span>
                 <span class="n">momentum</span><span class="o">=</span><span class="mf">0.9</span><span class="p">,</span>
                 <span class="n">affine</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
                 <span class="n">gamma_init</span><span class="o">=</span><span class="s1">&#39;ones&#39;</span><span class="p">,</span>
                 <span class="n">beta_init</span><span class="o">=</span><span class="s1">&#39;zeros&#39;</span><span class="p">,</span>
                 <span class="n">moving_mean_init</span><span class="o">=</span><span class="s1">&#39;zeros&#39;</span><span class="p">,</span>
                 <span class="n">moving_var_init</span><span class="o">=</span><span class="s1">&#39;ones&#39;</span><span class="p">,</span>
                 <span class="n">use_batch_statistics</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">device_num_each_group</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
                 <span class="n">process_groups</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
                 <span class="n">input_dims</span><span class="o">=</span><span class="s1">&#39;2d&#39;</span><span class="p">,</span>
                 <span class="n">data_format</span><span class="o">=</span><span class="s1">&#39;NCHW&#39;</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize _BatchNorm.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">_BatchNorm</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="n">validator</span><span class="o">.</span><span class="n">check_value_type</span><span class="p">(</span><span class="s1">&#39;num_features&#39;</span><span class="p">,</span> <span class="n">num_features</span><span class="p">,</span> <span class="p">[</span><span class="nb">int</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">num_features</span> <span class="o">&lt;</span> <span class="mi">1</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="si">}</span><span class="s2">&#39;, the &#39;num_features&#39; must be at least 1, but got </span><span class="si">{</span><span class="n">num_features</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>

        <span class="k">if</span> <span class="n">momentum</span> <span class="o">&lt;</span> <span class="mi">0</span> <span class="ow">or</span> <span class="n">momentum</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="si">}</span><span class="s2">&#39;, the &#39;momentum&#39; should be a number in range [0, 1], &quot;</span>
                             <span class="sa">f</span><span class="s2">&quot;but got </span><span class="si">{</span><span class="n">momentum</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">input_dims</span> <span class="o">=</span> <span class="n">input_dims</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">format</span> <span class="o">=</span> <span class="n">validator</span><span class="o">.</span><span class="n">check_string</span><span class="p">(</span><span class="n">data_format</span><span class="p">,</span> <span class="p">[</span><span class="s1">&#39;NCHW&#39;</span><span class="p">,</span> <span class="s1">&#39;NHWC&#39;</span><span class="p">],</span> <span class="s1">&#39;format&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">context</span><span class="o">.</span><span class="n">get_context</span><span class="p">(</span><span class="s2">&quot;device_target&quot;</span><span class="p">)</span> <span class="o">!=</span> <span class="s2">&quot;GPU&quot;</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">format</span> <span class="o">==</span> <span class="s2">&quot;NHWC&quot;</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="si">}</span><span class="s2">&#39;, the &#39;NHWC&#39; format only support in GPU target, but got device &quot;</span>
                             <span class="sa">f</span><span class="s2">&quot;target </span><span class="si">{</span><span class="n">context</span><span class="o">.</span><span class="n">get_context</span><span class="p">(</span><span class="s1">&#39;device_target&#39;</span><span class="p">)</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">use_batch_statistics</span> <span class="o">=</span> <span class="n">use_batch_statistics</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_batch_statistics</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">use_batch_statistics</span><span class="p">,</span> <span class="nb">bool</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="si">}</span><span class="s2">&#39;, the &#39;use_batch_statistics&#39; should be a boolean value or None,&quot;</span>
                             <span class="sa">f</span><span class="s2">&quot; but got </span><span class="si">{</span><span class="n">use_batch_statistics</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">num_features</span> <span class="o">=</span> <span class="n">num_features</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">eps</span> <span class="o">=</span> <span class="n">eps</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">moving_mean</span> <span class="o">=</span> <span class="n">Parameter</span><span class="p">(</span><span class="n">initializer</span><span class="p">(</span>
            <span class="n">moving_mean_init</span><span class="p">,</span> <span class="n">num_features</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;mean&quot;</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">moving_variance</span> <span class="o">=</span> <span class="n">Parameter</span><span class="p">(</span><span class="n">initializer</span><span class="p">(</span>
            <span class="n">moving_var_init</span><span class="p">,</span> <span class="n">num_features</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;variance&quot;</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="n">Parameter</span><span class="p">(</span><span class="n">initializer</span><span class="p">(</span>
            <span class="n">gamma_init</span><span class="p">,</span> <span class="n">num_features</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;gamma&quot;</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="n">affine</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">=</span> <span class="n">Parameter</span><span class="p">(</span><span class="n">initializer</span><span class="p">(</span>
            <span class="n">beta_init</span><span class="p">,</span> <span class="n">num_features</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;beta&quot;</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="n">affine</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">group_device_num</span> <span class="o">=</span> <span class="n">validator</span><span class="o">.</span><span class="n">check_positive_int</span><span class="p">(</span><span class="n">device_num_each_group</span><span class="p">,</span> <span class="s2">&quot;device_num_each_group&quot;</span><span class="p">,</span>
                                                             <span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">process_groups</span> <span class="o">=</span> <span class="n">process_groups</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">is_global</span> <span class="o">=</span> <span class="kc">False</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">parallel_mode</span> <span class="o">=</span> <span class="n">context</span><span class="o">.</span><span class="n">get_auto_parallel_context</span><span class="p">(</span><span class="s2">&quot;parallel_mode&quot;</span><span class="p">)</span>
        <span class="k">global</span> <span class="n">SYNC_BN_GROUP_NAME</span>
        <span class="c1"># for GlobalBatchNorm</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">group_device_num</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">rank_id</span> <span class="o">=</span> <span class="n">get_rank</span><span class="p">()</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">rank_size</span> <span class="o">=</span> <span class="n">get_group_size</span><span class="p">()</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">device_list</span> <span class="o">=</span> <span class="p">[</span><span class="n">i</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">rank_size</span><span class="p">)]</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">rank_list</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">list_group</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device_list</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">group_device_num</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">rank_list_idx</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">rank_list</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_create_global_groups</span><span class="p">()</span>
        <span class="c1"># for SyncBatchNorm</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">process_groups</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">rank_id</span> <span class="o">=</span> <span class="n">get_rank</span><span class="p">()</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">rank_size</span> <span class="o">=</span> <span class="n">get_group_size</span><span class="p">()</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">process_groups</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="n">validator</span><span class="o">.</span><span class="n">check_isinstance</span><span class="p">(</span><span class="s2">&quot;process_groups&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">process_groups</span><span class="p">,</span> <span class="nb">list</span><span class="p">)</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_check_rank_ids</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">process_groups</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">rank_size</span><span class="p">)</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_create_sync_groups</span><span class="p">()</span>
            <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">rank_size</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">is_global</span> <span class="o">=</span> <span class="kc">True</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">group_device_num</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">rank_size</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">device_list</span> <span class="o">=</span> <span class="p">[</span><span class="n">i</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">rank_size</span><span class="p">)]</span>
                <span class="k">if</span> <span class="n">context</span><span class="o">.</span><span class="n">get_context</span><span class="p">(</span><span class="s2">&quot;device_target&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="s2">&quot;Ascend&quot;</span><span class="p">:</span>
                    <span class="k">if</span> <span class="n">SYNC_BN_GROUP_NAME</span> <span class="o">==</span> <span class="s2">&quot;&quot;</span><span class="p">:</span>
                        <span class="n">SYNC_BN_GROUP_NAME</span> <span class="o">=</span> <span class="s2">&quot;sync_bn_group0&quot;</span>
                        <span class="n">management</span><span class="o">.</span><span class="n">create_group</span><span class="p">(</span><span class="n">SYNC_BN_GROUP_NAME</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">device_list</span><span class="p">)</span>
                <span class="k">elif</span> <span class="n">context</span><span class="o">.</span><span class="n">get_context</span><span class="p">(</span><span class="s2">&quot;device_target&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="s2">&quot;GPU&quot;</span><span class="p">:</span>
                    <span class="k">if</span> <span class="n">SYNC_BN_GROUP_NAME</span> <span class="o">==</span> <span class="s2">&quot;&quot;</span><span class="p">:</span>
                        <span class="n">SYNC_BN_GROUP_NAME</span> <span class="o">=</span> <span class="s2">&quot;nccl_world_group&quot;</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">shape</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Shape</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">reduce_mean</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">ReduceMean</span><span class="p">(</span><span class="n">keep_dims</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">square</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Square</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">sqrt</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Sqrt</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">cast</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Cast</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">dtype</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">DType</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">reshape</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Reshape</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_target</span> <span class="o">=</span> <span class="n">context</span><span class="o">.</span><span class="n">get_context</span><span class="p">(</span><span class="s2">&quot;device_target&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">is_graph_mode</span> <span class="o">=</span> <span class="n">context</span><span class="o">.</span><span class="n">get_context</span><span class="p">(</span><span class="s2">&quot;mode&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="n">context</span><span class="o">.</span><span class="n">GRAPH_MODE</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">momentum</span> <span class="o">=</span> <span class="mf">1.0</span> <span class="o">-</span> <span class="n">momentum</span>
        <span class="k">if</span> <span class="n">context</span><span class="o">.</span><span class="n">get_context</span><span class="p">(</span><span class="s2">&quot;enable_ge&quot;</span><span class="p">):</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">is_ge_backend</span> <span class="o">=</span> <span class="kc">True</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">is_ge_backend</span> <span class="o">=</span> <span class="kc">False</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">bn_train</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">BatchNorm</span><span class="p">(</span><span class="n">is_training</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
                                    <span class="n">epsilon</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">eps</span><span class="p">,</span>
                                    <span class="n">momentum</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">momentum</span><span class="p">,</span>
                                    <span class="n">data_format</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">format</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_global</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">bn_train</span> <span class="o">=</span> <span class="n">inner</span><span class="o">.</span><span class="n">SyncBatchNorm</span><span class="p">(</span><span class="n">epsilon</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">eps</span><span class="p">,</span>
                                                <span class="n">momentum</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">momentum</span><span class="p">,</span>
                                                <span class="n">group</span><span class="o">=</span><span class="n">SYNC_BN_GROUP_NAME</span><span class="p">,</span>
                                                <span class="n">device_num</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">group_device_num</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">bn_infer</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">BatchNorm</span><span class="p">(</span><span class="n">is_training</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">epsilon</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">eps</span><span class="p">,</span> <span class="n">data_format</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">format</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">_is_in_auto_parallel_mode</span><span class="p">():</span>
            <span class="n">data_parallel_strategy</span> <span class="o">=</span> <span class="p">((</span><span class="mi">1</span><span class="p">,),</span> <span class="p">(</span><span class="mi">1</span><span class="p">,))</span>
            <span class="n">data_parallel_strategy_one</span> <span class="o">=</span> <span class="p">((</span><span class="mi">1</span><span class="p">,),</span> <span class="p">())</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">data_parallel_strategy</span> <span class="o">=</span> <span class="kc">None</span>
            <span class="n">data_parallel_strategy_one</span> <span class="o">=</span> <span class="kc">None</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">sub_mean</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Sub</span><span class="p">()</span><span class="o">.</span><span class="n">shard</span><span class="p">(</span><span class="n">data_parallel_strategy</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">sub_var</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Sub</span><span class="p">()</span><span class="o">.</span><span class="n">shard</span><span class="p">(</span><span class="n">data_parallel_strategy</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">mul_mean</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Mul</span><span class="p">()</span><span class="o">.</span><span class="n">shard</span><span class="p">(</span><span class="n">data_parallel_strategy_one</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">mul_var</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Mul</span><span class="p">()</span><span class="o">.</span><span class="n">shard</span><span class="p">(</span><span class="n">data_parallel_strategy_one</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">assign_sub_mean</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">AssignSub</span><span class="p">()</span><span class="o">.</span><span class="n">shard</span><span class="p">(</span><span class="n">data_parallel_strategy</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">assign_sub_var</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">AssignSub</span><span class="p">()</span><span class="o">.</span><span class="n">shard</span><span class="p">(</span><span class="n">data_parallel_strategy</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_check_data_dim</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="k">raise</span> <span class="ne">NotImplementedError</span>

    <span class="k">def</span> <span class="nf">list_group</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">world_rank</span><span class="p">,</span> <span class="n">group_size</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot; Check whether world_rank and group_size  are valid. &quot;&quot;&quot;</span>
        <span class="k">if</span> <span class="n">group_size</span> <span class="o">&gt;</span> <span class="n">get_group_size</span><span class="p">():</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="si">}</span><span class="s2">&#39;, the &#39;device_num_each_group&#39; cannot be greater than &quot;</span>
                             <span class="sa">f</span><span class="s2">&quot;local rank size, but got &#39;device_num_each_group&#39;: </span><span class="si">{</span><span class="n">group_size</span><span class="si">}</span><span class="s2">, &quot;</span>
                             <span class="sa">f</span><span class="s2">&quot;local rank size: </span><span class="si">{</span><span class="n">get_group_size</span><span class="p">()</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">world_rank</span><span class="p">)</span> <span class="o">%</span> <span class="n">group_size</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="si">}</span><span class="s2">&#39;, the dimension of device_list should be divisible by &quot;</span>
                             <span class="sa">f</span><span class="s2">&quot;&#39;device_num_each_group&#39;, but got the length of device_list: </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">world_rank</span><span class="p">)</span><span class="si">}</span><span class="s2">, &quot;</span>
                             <span class="sa">f</span><span class="s2">&quot;&#39;device_num_each_group&#39;: </span><span class="si">{</span><span class="n">group_size</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>
        <span class="n">world_rank_list</span> <span class="o">=</span> <span class="nb">zip</span><span class="p">(</span><span class="o">*</span><span class="p">(</span><span class="nb">iter</span><span class="p">(</span><span class="n">world_rank</span><span class="p">),)</span> <span class="o">*</span> <span class="n">group_size</span><span class="p">)</span>
        <span class="n">group_list</span> <span class="o">=</span> <span class="p">[</span><span class="nb">list</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">world_rank_list</span><span class="p">]</span>
        <span class="k">return</span> <span class="n">group_list</span>

    <span class="k">def</span> <span class="nf">_check_rank_ids</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">process_groups</span><span class="p">,</span> <span class="n">rank_size</span><span class="p">):</span>
        <span class="n">seen</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
        <span class="k">for</span> <span class="n">rid</span> <span class="ow">in</span> <span class="n">itertools</span><span class="o">.</span><span class="n">chain</span><span class="p">(</span><span class="o">*</span><span class="n">process_groups</span><span class="p">):</span>
            <span class="n">validator</span><span class="o">.</span><span class="n">check_int_range</span><span class="p">(</span><span class="n">rid</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">rank_size</span><span class="p">,</span> <span class="n">Rel</span><span class="o">.</span><span class="n">INC_LEFT</span><span class="p">,</span> <span class="s2">&quot;rank id in process_groups&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="p">)</span>
            <span class="k">if</span> <span class="n">rid</span> <span class="ow">in</span> <span class="n">seen</span><span class="p">:</span>
                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="si">}</span><span class="s2">&#39;, rank id in &#39;process_groups&#39; should not be duplicated, &quot;</span>
                                 <span class="sa">f</span><span class="s2">&quot;but got </span><span class="si">{</span><span class="n">process_groups</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>
            <span class="n">seen</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">rid</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_create_global_groups</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">rank_list_idx</span><span class="p">):</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">rank_id</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">rank_list</span><span class="p">[</span><span class="n">i</span><span class="p">]:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">is_global</span> <span class="o">=</span> <span class="kc">True</span>
                <span class="k">global</span> <span class="n">SYNC_BN_GROUP_NAME</span>
                <span class="k">if</span> <span class="n">SYNC_BN_GROUP_NAME</span> <span class="o">==</span> <span class="s2">&quot;&quot;</span><span class="p">:</span>
                    <span class="n">SYNC_BN_GROUP_NAME</span> <span class="o">=</span> <span class="s2">&quot;sync_bn_group&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">)</span>
                    <span class="n">management</span><span class="o">.</span><span class="n">create_group</span><span class="p">(</span><span class="n">SYNC_BN_GROUP_NAME</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">rank_list</span><span class="p">[</span><span class="n">i</span><span class="p">])</span>

    <span class="k">def</span> <span class="nf">_create_sync_groups</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">process_groups</span><span class="p">)):</span>
            <span class="n">validator</span><span class="o">.</span><span class="n">check_isinstance</span><span class="p">(</span><span class="s2">&quot;process_groups[&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> <span class="o">+</span> <span class="s2">&quot;]&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">process_groups</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="nb">list</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">group_device_num</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">process_groups</span><span class="p">[</span><span class="n">i</span><span class="p">])</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">rank_id</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">process_groups</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">group_device_num</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">is_global</span> <span class="o">=</span> <span class="kc">True</span>
                <span class="k">global</span> <span class="n">SYNC_BN_GROUP_NAME</span>
                <span class="k">if</span> <span class="n">SYNC_BN_GROUP_NAME</span> <span class="o">==</span> <span class="s2">&quot;&quot;</span><span class="p">:</span>
                    <span class="n">SYNC_BN_GROUP_NAME</span> <span class="o">=</span> <span class="s2">&quot;sync_bn_group&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">)</span>
                    <span class="n">management</span><span class="o">.</span><span class="n">create_group</span><span class="p">(</span><span class="n">SYNC_BN_GROUP_NAME</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">process_groups</span><span class="p">[</span><span class="n">i</span><span class="p">])</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="n">_shape_check_bn</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">shape</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">input_dims</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_batch_statistics</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">training</span><span class="p">:</span>
                <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">bn_train</span><span class="p">(</span><span class="n">x</span><span class="p">,</span>
                                     <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span><span class="p">,</span>
                                     <span class="bp">self</span><span class="o">.</span><span class="n">beta</span><span class="p">,</span>
                                     <span class="bp">self</span><span class="o">.</span><span class="n">moving_mean</span><span class="p">,</span>
                                     <span class="bp">self</span><span class="o">.</span><span class="n">moving_variance</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
            <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">training</span><span class="p">:</span>
                <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">bn_infer</span><span class="p">(</span><span class="n">x</span><span class="p">,</span>
                                     <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span><span class="p">,</span>
                                     <span class="bp">self</span><span class="o">.</span><span class="n">beta</span><span class="p">,</span>
                                     <span class="bp">self</span><span class="o">.</span><span class="n">moving_mean</span><span class="p">,</span>
                                     <span class="bp">self</span><span class="o">.</span><span class="n">moving_variance</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_batch_statistics</span> <span class="ow">is</span> <span class="kc">True</span><span class="p">:</span>
            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">bn_train</span><span class="p">(</span><span class="n">x</span><span class="p">,</span>
                                 <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span><span class="p">,</span>
                                 <span class="bp">self</span><span class="o">.</span><span class="n">beta</span><span class="p">,</span>
                                 <span class="bp">self</span><span class="o">.</span><span class="n">moving_mean</span><span class="p">,</span>
                                 <span class="bp">self</span><span class="o">.</span><span class="n">moving_variance</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>

        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">bn_infer</span><span class="p">(</span><span class="n">x</span><span class="p">,</span>
                             <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span><span class="p">,</span>
                             <span class="bp">self</span><span class="o">.</span><span class="n">beta</span><span class="p">,</span>
                             <span class="bp">self</span><span class="o">.</span><span class="n">moving_mean</span><span class="p">,</span>
                             <span class="bp">self</span><span class="o">.</span><span class="n">moving_variance</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>

    <span class="k">def</span> <span class="nf">extend_repr</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="s1">&#39;num_features=</span><span class="si">{}</span><span class="s1">, eps=</span><span class="si">{}</span><span class="s1">, momentum=</span><span class="si">{}</span><span class="s1">, gamma=</span><span class="si">{}</span><span class="s1">, beta=</span><span class="si">{}</span><span class="s1">, moving_mean=</span><span class="si">{}</span><span class="s1">, moving_variance=</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">num_features</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">eps</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">momentum</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">beta</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">moving_mean</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">moving_variance</span><span class="p">)</span>


<span class="nd">@constexpr</span>
<span class="k">def</span> <span class="nf">_channel_check</span><span class="p">(</span><span class="n">channel</span><span class="p">,</span> <span class="n">num_channel</span><span class="p">,</span> <span class="n">prim_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="n">msg_prefix</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="n">prim_name</span><span class="si">}</span><span class="s2">&#39;, the&quot;</span> <span class="k">if</span> <span class="n">prim_name</span> <span class="k">else</span> <span class="s2">&quot;The&quot;</span>
    <span class="k">if</span> <span class="n">channel</span> <span class="o">!=</span> <span class="n">num_channel</span><span class="p">:</span>
        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">msg_prefix</span><span class="si">}</span><span class="s2"> channel(the second dim of the input &#39;x&#39;) should be equal to num_channels, &quot;</span>
                         <span class="sa">f</span><span class="s2">&quot;but got channel: </span><span class="si">{</span><span class="n">channel</span><span class="si">}</span><span class="s2">, num_channels: </span><span class="si">{</span><span class="n">num_channel</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>


<span class="nd">@constexpr</span>
<span class="k">def</span> <span class="nf">_shape_check</span><span class="p">(</span><span class="n">in_shape</span><span class="p">,</span> <span class="n">prim_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="n">msg_prefix</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="n">prim_name</span><span class="si">}</span><span class="s2">&#39;, the&quot;</span> <span class="k">if</span> <span class="n">prim_name</span> <span class="k">else</span> <span class="s2">&quot;The&quot;</span>
    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">in_shape</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">4</span><span class="p">:</span>
        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">msg_prefix</span><span class="si">}</span><span class="s2"> in_shape must has 4 dims, but got the length of in_shape: </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">in_shape</span><span class="p">)</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>


<span class="nd">@constexpr</span>
<span class="k">def</span> <span class="nf">_shape_check_bn</span><span class="p">(</span><span class="n">in_shape</span><span class="p">,</span> <span class="n">in_dims</span><span class="p">,</span> <span class="n">prim_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;check input dims of batch norm.&quot;&quot;&quot;</span>
    <span class="n">msg_prefix</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="n">prim_name</span><span class="si">}</span><span class="s2">&#39;, the&quot;</span> <span class="k">if</span> <span class="n">prim_name</span> <span class="k">else</span> <span class="s2">&quot;The&quot;</span>
    <span class="n">dim</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">in_shape</span><span class="p">)</span>
    <span class="k">if</span> <span class="n">in_dims</span> <span class="o">==</span> <span class="s1">&#39;1d&#39;</span> <span class="ow">and</span> <span class="n">dim</span> <span class="o">!=</span> <span class="mi">2</span><span class="p">:</span>
        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">msg_prefix</span><span class="si">}</span><span class="s2"> in_shape must have 2 dims, but got </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">in_shape</span><span class="p">)</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>
    <span class="k">if</span> <span class="n">in_dims</span> <span class="o">==</span> <span class="s1">&#39;2d&#39;</span> <span class="ow">and</span> <span class="n">dim</span> <span class="o">!=</span> <span class="mi">4</span><span class="p">:</span>
        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">msg_prefix</span><span class="si">}</span><span class="s2"> in_shape must have 4 dims, but got </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">in_shape</span><span class="p">)</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>
    <span class="k">if</span> <span class="n">in_dims</span> <span class="o">==</span> <span class="s1">&#39;3d&#39;</span> <span class="ow">and</span> <span class="n">dim</span> <span class="o">!=</span> <span class="mi">5</span><span class="p">:</span>
        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">msg_prefix</span><span class="si">}</span><span class="s2"> in_shape must have 5 dims, but got </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">in_shape</span><span class="p">)</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>
    <span class="k">if</span> <span class="n">in_dims</span> <span class="o">==</span> <span class="s1">&#39;both&#39;</span> <span class="ow">and</span> <span class="n">dim</span> <span class="o">!=</span> <span class="mi">2</span> <span class="ow">and</span> <span class="n">dim</span> <span class="o">!=</span> <span class="mi">4</span><span class="p">:</span>
        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">msg_prefix</span><span class="si">}</span><span class="s2"> in_shape must have 2 dims or 4 dims, but got </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">in_shape</span><span class="p">)</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>


<span class="nd">@constexpr</span>
<span class="k">def</span> <span class="nf">_shape_infer</span><span class="p">(</span><span class="n">x_shape</span><span class="p">,</span> <span class="n">num_feature</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;global Batch Normalization shape and axes infer&quot;&quot;&quot;</span>
    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">x_shape</span><span class="p">)</span> <span class="o">==</span> <span class="mi">4</span><span class="p">:</span>
        <span class="n">axes</span> <span class="o">=</span> <span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
        <span class="n">re_shape</span> <span class="o">=</span> <span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">num_feature</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="n">axes</span> <span class="o">=</span> <span class="p">(</span><span class="mi">0</span><span class="p">,)</span>
        <span class="n">re_shape</span> <span class="o">=</span> <span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">num_feature</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">axes</span><span class="p">,</span> <span class="n">re_shape</span>


<span class="k">class</span> <span class="nc">BatchNorm1d</span><span class="p">(</span><span class="n">_BatchNorm</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Batch Normalization layer over a 2D input.</span>

<span class="sd">    Batch Normalization is widely used in convolutional networks. This layer</span>
<span class="sd">    applies Batch Normalization over a 2D input (a mini-batch of 1D inputs) to</span>
<span class="sd">    reduce internal covariate shift as described in the paper</span>
<span class="sd">    `Batch Normalization: Accelerating Deep Network Training by</span>
<span class="sd">    Reducing Internal Covariate Shift &lt;https://arxiv.org/abs/1502.03167&gt;`_. It</span>
<span class="sd">    rescales and recenters the feature using a mini-batch of data and</span>
<span class="sd">    the learned parameters which can be described in the following formula.</span>

<span class="sd">    .. math::</span>
<span class="sd">        y = \frac{x - \mathrm{E}[x]}{\sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta</span>

<span class="sd">    Note:</span>
<span class="sd">        The implementation of BatchNorm is different in graph mode and pynative mode, therefore the mode is not</span>
<span class="sd">        recommended to be changed after net was initialized.</span>

<span class="sd">    Args:</span>
<span class="sd">        num_features (int): `C` from an expected input of size (N, C).</span>
<span class="sd">        eps (float): A value added to the denominator for numerical stability. Default: 1e-5.</span>
<span class="sd">        momentum (float): A floating hyperparameter of the momentum for the</span>
<span class="sd">            running_mean and running_var computation. Default: 0.9.</span>
<span class="sd">        affine (bool): A bool value. When set to True, gamma and beta can be learned. Default: True.</span>
<span class="sd">        gamma_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the gamma weight.</span>
<span class="sd">            The values of str refer to the function `initializer` including &#39;zeros&#39;, &#39;ones&#39;, etc. Default: &#39;ones&#39;.</span>
<span class="sd">        beta_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the beta weight.</span>
<span class="sd">            The values of str refer to the function `initializer` including &#39;zeros&#39;, &#39;ones&#39;, etc. Default: &#39;zeros&#39;.</span>
<span class="sd">        moving_mean_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the moving mean.</span>
<span class="sd">            The values of str refer to the function `initializer` including &#39;zeros&#39;, &#39;ones&#39;, etc. Default: &#39;zeros&#39;.</span>
<span class="sd">        moving_var_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the moving variance.</span>
<span class="sd">            The values of str refer to the function `initializer` including &#39;zeros&#39;, &#39;ones&#39;, etc. Default: &#39;ones&#39;.</span>
<span class="sd">        use_batch_statistics (bool): If true, use the mean value and variance value of current batch data. If false,</span>
<span class="sd">            use the mean value and variance value of specified value. If None, the training process will use the mean</span>
<span class="sd">            and variance of current batch data and track the running mean and variance, the evaluation process will use</span>
<span class="sd">            the running mean and variance. Default: None.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **x** (Tensor) - Tensor of shape :math:`(N, C_{in})`.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, the normalized, scaled, offset tensor, of shape :math:`(N, C_{out})`.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU`` ``CPU``</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If `num_features` is not an int.</span>
<span class="sd">        TypeError: If `eps` is not a float.</span>
<span class="sd">        ValueError: If `num_features` is less than 1.</span>
<span class="sd">        ValueError: If `momentum` is not in range [0, 1].</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; import numpy as np</span>
<span class="sd">        &gt;&gt;&gt; import mindspore.nn as nn</span>
<span class="sd">        &gt;&gt;&gt; from mindspore import Tensor</span>
<span class="sd">        &gt;&gt;&gt; net = nn.BatchNorm1d(num_features=4)</span>
<span class="sd">        &gt;&gt;&gt; x = Tensor(np.array([[0.7, 0.5, 0.5, 0.6],</span>
<span class="sd">        ...                      [0.5, 0.4, 0.6, 0.9]]).astype(np.float32))</span>
<span class="sd">        &gt;&gt;&gt; output = net(x)</span>
<span class="sd">        &gt;&gt;&gt; print(output)</span>
<span class="sd">        [[ 0.6999965   0.4999975  0.4999975  0.59999704 ]</span>
<span class="sd">         [ 0.4999975   0.399998   0.59999704 0.89999545 ]]</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
                 <span class="n">num_features</span><span class="p">,</span>
                 <span class="n">eps</span><span class="o">=</span><span class="mf">1e-5</span><span class="p">,</span>
                 <span class="n">momentum</span><span class="o">=</span><span class="mf">0.9</span><span class="p">,</span>
                 <span class="n">affine</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
                 <span class="n">gamma_init</span><span class="o">=</span><span class="s1">&#39;ones&#39;</span><span class="p">,</span>
                 <span class="n">beta_init</span><span class="o">=</span><span class="s1">&#39;zeros&#39;</span><span class="p">,</span>
                 <span class="n">moving_mean_init</span><span class="o">=</span><span class="s1">&#39;zeros&#39;</span><span class="p">,</span>
                 <span class="n">moving_var_init</span><span class="o">=</span><span class="s1">&#39;ones&#39;</span><span class="p">,</span>
                 <span class="n">use_batch_statistics</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize BatchNorm1d.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">BatchNorm1d</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_features</span><span class="p">,</span>
                                          <span class="n">eps</span><span class="p">,</span>
                                          <span class="n">momentum</span><span class="p">,</span>
                                          <span class="n">affine</span><span class="p">,</span>
                                          <span class="n">gamma_init</span><span class="p">,</span>
                                          <span class="n">beta_init</span><span class="p">,</span>
                                          <span class="n">moving_mean_init</span><span class="p">,</span>
                                          <span class="n">moving_var_init</span><span class="p">,</span>
                                          <span class="n">use_batch_statistics</span><span class="p">,</span>
                                          <span class="n">input_dims</span><span class="o">=</span><span class="s1">&#39;1d&#39;</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_check_data_dim</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="k">if</span> <span class="n">x</span><span class="o">.</span><span class="n">ndim</span> <span class="o">!=</span> <span class="mi">2</span><span class="p">:</span>
            <span class="k">pass</span>


<span class="k">class</span> <span class="nc">BatchNorm2d</span><span class="p">(</span><span class="n">_BatchNorm</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Batch Normalization layer over a 4D input.</span>

<span class="sd">    Batch Normalization is widely used in convolutional networks. This layer</span>
<span class="sd">    applies Batch Normalization over a 4D input (a mini-batch of 2D inputs with</span>
<span class="sd">    additional channel dimension) to avoid internal covariate shift as described</span>
<span class="sd">    in the paper `Batch Normalization: Accelerating Deep Network Training by</span>
<span class="sd">    Reducing Internal Covariate Shift &lt;https://arxiv.org/abs/1502.03167&gt;`_. It</span>
<span class="sd">    rescales and recenters the feature using a mini-batch of data and</span>
<span class="sd">    the learned parameters which can be described in the following formula.</span>

<span class="sd">    .. math::</span>
<span class="sd">        y = \frac{x - \mathrm{E}[x]}{\sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta</span>

<span class="sd">    Note:</span>
<span class="sd">        The implementation of BatchNorm is different in graph mode and pynative mode, therefore that mode can not be</span>
<span class="sd">        changed after net was initialized.</span>
<span class="sd">        Note that the formula for updating the moving_mean and moving_var is</span>

<span class="sd">        .. math::</span>
<span class="sd">            \text{moving_mean}=\text{moving_mean∗momentum}+μ_β\text{∗(1−momentum)}\\</span>
<span class="sd">            \text{moving_var}=\text{moving_var∗momentum}+σ^2_β\text{∗(1−momentum)}</span>

<span class="sd">        where :math:`moving_mean, moving_var` are the updated mean and variance,</span>
<span class="sd">        :math:`μ_β, σ^2_β` are the observed value (mean and variance) of each batch of data.</span>

<span class="sd">    Args:</span>
<span class="sd">        num_features (int): The number of channels of the input tensor. Expected input size is (N, C, H, W),</span>
<span class="sd">            `C` represents the number of channels</span>
<span class="sd">        eps (float): A value added to the denominator for numerical stability. Default: 1e-5.</span>
<span class="sd">        momentum (float): A floating hyperparameter of the momentum for the</span>
<span class="sd">            running_mean and running_var computation. Default: 0.9.</span>
<span class="sd">        affine (bool): A bool value. When set to True, gamma and beta can be learned. Default: True.</span>
<span class="sd">        gamma_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the gamma weight.</span>
<span class="sd">            The values of str refer to the function `initializer` including &#39;zeros&#39;, &#39;ones&#39;, etc. Default: &#39;ones&#39;.</span>
<span class="sd">        beta_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the beta weight.</span>
<span class="sd">            The values of str refer to the function `initializer` including &#39;zeros&#39;, &#39;ones&#39;, etc. Default: &#39;zeros&#39;.</span>
<span class="sd">        moving_mean_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the moving mean.</span>
<span class="sd">            The values of str refer to the function `initializer` including &#39;zeros&#39;, &#39;ones&#39;, etc. Default: &#39;zeros&#39;.</span>
<span class="sd">        moving_var_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the moving variance.</span>
<span class="sd">            The values of str refer to the function `initializer` including &#39;zeros&#39;, &#39;ones&#39;, etc. Default: &#39;ones&#39;.</span>
<span class="sd">        use_batch_statistics (bool):</span>

<span class="sd">            - If true, use the mean value and variance value of current batch data and track running mean</span>
<span class="sd">              and running variance.</span>
<span class="sd">            - If false, use the mean value and variance value of specified value, and not track statistical value.</span>
<span class="sd">            - If None, the use_batch_statistics is automatically set to true or false according to the training</span>
<span class="sd">              and evaluation mode. During training, the parameter is set to true, and during evaluation, the</span>
<span class="sd">              parameter is set to false. Default: None.</span>

<span class="sd">        data_format (str): The optional value for data format, is &#39;NHWC&#39; or &#39;NCHW&#39;.</span>
<span class="sd">            Default: &#39;NCHW&#39;.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **x** (Tensor) - Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})`.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, the normalized, scaled, offset tensor, of shape :math:`(N, C_{out}, H_{out}, W_{out})`.</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If `num_features` is not an int.</span>
<span class="sd">        TypeError: If `eps` is not a float.</span>
<span class="sd">        ValueError: If `num_features` is less than 1.</span>
<span class="sd">        ValueError: If `momentum` is not in range [0, 1].</span>
<span class="sd">        ValueError: If `data_format` is neither &#39;NHWC&#39; not &#39;NCHW&#39;.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU`` ``CPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; import numpy as np</span>
<span class="sd">        &gt;&gt;&gt; import mindspore.nn as nn</span>
<span class="sd">        &gt;&gt;&gt; from mindspore import Tensor</span>
<span class="sd">        &gt;&gt;&gt; net = nn.BatchNorm2d(num_features=3)</span>
<span class="sd">        &gt;&gt;&gt; x = Tensor(np.ones([1, 3, 2, 2]).astype(np.float32))</span>
<span class="sd">        &gt;&gt;&gt; output = net(x)</span>
<span class="sd">        &gt;&gt;&gt; print(output)</span>
<span class="sd">        [[[[ 0.999995 0.999995 ]</span>
<span class="sd">           [ 0.999995 0.999995 ]]</span>
<span class="sd">          [[ 0.999995 0.999995 ]</span>
<span class="sd">           [ 0.999995 0.999995 ]]</span>
<span class="sd">          [[ 0.999995 0.999995 ]</span>
<span class="sd">           [ 0.999995 0.999995 ]]]]</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
                 <span class="n">num_features</span><span class="p">,</span>
                 <span class="n">eps</span><span class="o">=</span><span class="mf">1e-5</span><span class="p">,</span>
                 <span class="n">momentum</span><span class="o">=</span><span class="mf">0.9</span><span class="p">,</span>
                 <span class="n">affine</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
                 <span class="n">gamma_init</span><span class="o">=</span><span class="s1">&#39;ones&#39;</span><span class="p">,</span>
                 <span class="n">beta_init</span><span class="o">=</span><span class="s1">&#39;zeros&#39;</span><span class="p">,</span>
                 <span class="n">moving_mean_init</span><span class="o">=</span><span class="s1">&#39;zeros&#39;</span><span class="p">,</span>
                 <span class="n">moving_var_init</span><span class="o">=</span><span class="s1">&#39;ones&#39;</span><span class="p">,</span>
                 <span class="n">use_batch_statistics</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">data_format</span><span class="o">=</span><span class="s1">&#39;NCHW&#39;</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize BatchNorm2d.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">BatchNorm2d</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_features</span><span class="p">,</span>
                                          <span class="n">eps</span><span class="p">,</span>
                                          <span class="n">momentum</span><span class="p">,</span>
                                          <span class="n">affine</span><span class="p">,</span>
                                          <span class="n">gamma_init</span><span class="p">,</span>
                                          <span class="n">beta_init</span><span class="p">,</span>
                                          <span class="n">moving_mean_init</span><span class="p">,</span>
                                          <span class="n">moving_var_init</span><span class="p">,</span>
                                          <span class="n">use_batch_statistics</span><span class="p">,</span>
                                          <span class="n">input_dims</span><span class="o">=</span><span class="s1">&#39;2d&#39;</span><span class="p">,</span>
                                          <span class="n">data_format</span><span class="o">=</span><span class="n">data_format</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_check_data_dim</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="k">if</span> <span class="n">x</span><span class="o">.</span><span class="n">ndim</span> <span class="o">!=</span> <span class="mi">4</span><span class="p">:</span>
            <span class="k">pass</span>


<span class="nd">@constexpr</span>
<span class="k">def</span> <span class="nf">_check_3d_shape</span><span class="p">(</span><span class="n">input_shape</span><span class="p">,</span> <span class="n">prim_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="n">msg_prefix</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="n">prim_name</span><span class="si">}</span><span class="s2">&#39;, the&quot;</span> <span class="k">if</span> <span class="n">prim_name</span> <span class="k">else</span> <span class="s2">&quot;The&quot;</span>
    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">input_shape</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">5</span><span class="p">:</span>
        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">msg_prefix</span><span class="si">}</span><span class="s2"> input_shape must be 5-dimensional, but got the length of input_shape: &quot;</span>
                         <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">input_shape</span><span class="p">)</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">BatchNorm3d</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Batch Normalization layer over a 5D input.</span>

<span class="sd">    Batch Normalization is widely used in convolutional networks. This layer</span>
<span class="sd">    applies Batch Normalization over a 5D input (a mini-batch of 3D inputs with</span>
<span class="sd">    additional channel dimension) to avoid internal covariate shift.</span>

<span class="sd">    .. math::</span>
<span class="sd">        y = \frac{x - \mathrm{E}[x]}{\sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta</span>

<span class="sd">    Note:</span>
<span class="sd">        The implementation of BatchNorm is different in graph mode and pynative mode, therefore that mode can not be</span>
<span class="sd">        changed after net was initialized.</span>
<span class="sd">        Note that the formula for updating the running_mean and running_var is</span>
<span class="sd">        :math:`\hat{x}_\text{new} = (1 - \text{momentum}) \times x_t + \text{momentum} \times \hat{x}`,</span>
<span class="sd">        where :math:`\hat{x}` is the estimated statistic and :math:`x_t` is the new observed value.</span>

<span class="sd">    Args:</span>
<span class="sd">        num_features (int): `C` from an expected input of size (N, C, D, H, W).</span>
<span class="sd">        eps (float): A value added to the denominator for numerical stability. Default: 1e-5.</span>
<span class="sd">        momentum (float): A floating hyperparameter of the momentum for the</span>
<span class="sd">            running_mean and running_var computation. Default: 0.9.</span>
<span class="sd">        affine (bool): A bool value. When set to True, gamma and beta can be learned. Default: True.</span>
<span class="sd">        gamma_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the gamma weight.</span>
<span class="sd">            The values of str refer to the function `initializer` including &#39;zeros&#39;, &#39;ones&#39;, etc. Default: &#39;ones&#39;.</span>
<span class="sd">        beta_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the beta weight.</span>
<span class="sd">            The values of str refer to the function `initializer` including &#39;zeros&#39;, &#39;ones&#39;, etc. Default: &#39;zeros&#39;.</span>
<span class="sd">        moving_mean_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the moving mean.</span>
<span class="sd">            The values of str refer to the function `initializer` including &#39;zeros&#39;, &#39;ones&#39;, etc. Default: &#39;zeros&#39;.</span>
<span class="sd">        moving_var_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the moving variance.</span>
<span class="sd">            The values of str refer to the function `initializer` including &#39;zeros&#39;, &#39;ones&#39;, etc. Default: &#39;ones&#39;.</span>
<span class="sd">        use_batch_statistics (bool): If true, use the mean value and variance value of current batch data. If false,</span>
<span class="sd">            use the mean value and variance value of specified value. If None, the training process will use the mean</span>
<span class="sd">            and variance of current batch data and track the running mean and variance, the evaluation process will use</span>
<span class="sd">            the running mean and variance. Default: None.</span>
<span class="sd">        data_format (str): The optional value for data format is &#39;NCDHW&#39;. Default: &#39;NCDHW&#39;.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **x** (Tensor) - Tensor of shape :math:`(N, C_{in}, D_{in}, H_{in}, W_{in})`.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, the normalized, scaled, offset tensor, of shape :math:`(N, C_{out}, D_{out},H_{out}, W_{out})`.</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If `num_features` is not an int.</span>
<span class="sd">        TypeError: If `eps` is not a float.</span>
<span class="sd">        ValueError: If `num_features` is less than 1.</span>
<span class="sd">        ValueError: If `momentum` is not in range [0, 1].</span>
<span class="sd">        ValueError: If `data_format` is not &#39;NCDHW&#39;.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU`` ``CPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; import numpy as np</span>
<span class="sd">        &gt;&gt;&gt; import mindspore.nn as nn</span>
<span class="sd">        &gt;&gt;&gt; from mindspore import Tensor</span>
<span class="sd">        &gt;&gt;&gt; net = nn.BatchNorm3d(num_features=3)</span>
<span class="sd">        &gt;&gt;&gt; x = Tensor(np.ones([16, 3, 10, 32, 32]).astype(np.float32))</span>
<span class="sd">        &gt;&gt;&gt; output = net(x)</span>
<span class="sd">        &gt;&gt;&gt; print(output.shape)</span>
<span class="sd">        (16, 3, 10, 32, 32)</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
                 <span class="n">num_features</span><span class="p">,</span>
                 <span class="n">eps</span><span class="o">=</span><span class="mf">1e-5</span><span class="p">,</span>
                 <span class="n">momentum</span><span class="o">=</span><span class="mf">0.9</span><span class="p">,</span>
                 <span class="n">affine</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
                 <span class="n">gamma_init</span><span class="o">=</span><span class="s1">&#39;ones&#39;</span><span class="p">,</span>
                 <span class="n">beta_init</span><span class="o">=</span><span class="s1">&#39;zeros&#39;</span><span class="p">,</span>
                 <span class="n">moving_mean_init</span><span class="o">=</span><span class="s1">&#39;zeros&#39;</span><span class="p">,</span>
                 <span class="n">moving_var_init</span><span class="o">=</span><span class="s1">&#39;ones&#39;</span><span class="p">,</span>
                 <span class="n">use_batch_statistics</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">data_format</span><span class="o">=</span><span class="s1">&#39;NCDHW&#39;</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize BatchNorm3d.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">BatchNorm3d</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">format</span> <span class="o">=</span> <span class="n">validator</span><span class="o">.</span><span class="n">check_string</span><span class="p">(</span><span class="n">data_format</span><span class="p">,</span> <span class="p">[</span><span class="s1">&#39;NCDHW&#39;</span><span class="p">],</span> <span class="s1">&#39;format&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">reshape</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Reshape</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">bn2d</span> <span class="o">=</span> <span class="n">BatchNorm2d</span><span class="p">(</span><span class="n">num_features</span><span class="o">=</span><span class="n">num_features</span><span class="p">,</span>
                                <span class="n">eps</span><span class="o">=</span><span class="n">eps</span><span class="p">,</span>
                                <span class="n">momentum</span><span class="o">=</span><span class="n">momentum</span><span class="p">,</span>
                                <span class="n">affine</span><span class="o">=</span><span class="n">affine</span><span class="p">,</span>
                                <span class="n">gamma_init</span><span class="o">=</span><span class="n">gamma_init</span><span class="p">,</span>
                                <span class="n">beta_init</span><span class="o">=</span><span class="n">beta_init</span><span class="p">,</span>
                                <span class="n">moving_mean_init</span><span class="o">=</span><span class="n">moving_mean_init</span><span class="p">,</span>
                                <span class="n">moving_var_init</span><span class="o">=</span><span class="n">moving_var_init</span><span class="p">,</span>
                                <span class="n">use_batch_statistics</span><span class="o">=</span><span class="n">use_batch_statistics</span><span class="p">,</span>
                                <span class="n">data_format</span><span class="o">=</span><span class="s2">&quot;NCHW&quot;</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">input_x</span><span class="p">):</span>
        <span class="n">x_shape</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">shape</span><span class="p">(</span><span class="n">input_x</span><span class="p">)</span>
        <span class="n">_check_3d_shape</span><span class="p">(</span><span class="n">x_shape</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="p">)</span>
        <span class="n">input_x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="n">input_x</span><span class="p">,</span> <span class="p">(</span><span class="n">x_shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">x_shape</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">x_shape</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span> <span class="o">*</span> <span class="n">x_shape</span><span class="p">[</span><span class="mi">3</span><span class="p">],</span> <span class="n">x_shape</span><span class="p">[</span><span class="mi">4</span><span class="p">]))</span>
        <span class="n">bn2d_out</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">bn2d</span><span class="p">(</span><span class="n">input_x</span><span class="p">)</span>
        <span class="n">bn3d_out</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="n">bn2d_out</span><span class="p">,</span> <span class="n">x_shape</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">bn3d_out</span>


<div class="viewcode-block" id="GlobalBatchNorm"><a class="viewcode-back" href="../../../../api_python/nn/mindspore.nn.GlobalBatchNorm.html#mindspore.nn.GlobalBatchNorm">[docs]</a><span class="k">class</span> <span class="nc">GlobalBatchNorm</span><span class="p">(</span><span class="n">_BatchNorm</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    The GlobalBatchNorm interface is deprecated, please use the :class:`mindspore.nn.SyncBatchNorm` instead.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        deprecated</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@deprecated</span><span class="p">(</span><span class="s2">&quot;1.2&quot;</span><span class="p">,</span> <span class="s2">&quot;SyncBatchNorm&quot;</span><span class="p">,</span> <span class="kc">True</span><span class="p">)</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
                 <span class="n">num_features</span><span class="p">,</span>
                 <span class="n">eps</span><span class="o">=</span><span class="mf">1e-5</span><span class="p">,</span>
                 <span class="n">momentum</span><span class="o">=</span><span class="mf">0.9</span><span class="p">,</span>
                 <span class="n">affine</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
                 <span class="n">gamma_init</span><span class="o">=</span><span class="s1">&#39;ones&#39;</span><span class="p">,</span>
                 <span class="n">beta_init</span><span class="o">=</span><span class="s1">&#39;zeros&#39;</span><span class="p">,</span>
                 <span class="n">moving_mean_init</span><span class="o">=</span><span class="s1">&#39;zeros&#39;</span><span class="p">,</span>
                 <span class="n">moving_var_init</span><span class="o">=</span><span class="s1">&#39;ones&#39;</span><span class="p">,</span>
                 <span class="n">use_batch_statistics</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">device_num_each_group</span><span class="o">=</span><span class="mi">2</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize GlobalBatchNorm.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">GlobalBatchNorm</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_features</span><span class="p">,</span>
                                              <span class="n">eps</span><span class="p">,</span>
                                              <span class="n">momentum</span><span class="p">,</span>
                                              <span class="n">affine</span><span class="p">,</span>
                                              <span class="n">gamma_init</span><span class="p">,</span>
                                              <span class="n">beta_init</span><span class="p">,</span>
                                              <span class="n">moving_mean_init</span><span class="p">,</span>
                                              <span class="n">moving_var_init</span><span class="p">,</span>
                                              <span class="n">use_batch_statistics</span><span class="p">,</span>
                                              <span class="n">device_num_each_group</span><span class="p">,</span>
                                              <span class="n">input_dims</span><span class="o">=</span><span class="s1">&#39;both&#39;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">group_device_num</span> <span class="o">=</span> <span class="n">validator</span><span class="o">.</span><span class="n">check_positive_int</span><span class="p">(</span><span class="n">device_num_each_group</span><span class="p">,</span> <span class="s2">&quot;device_num_each_group&quot;</span><span class="p">,</span>
                                                             <span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">group_device_num</span> <span class="o">&lt;=</span> <span class="mi">1</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="si">}</span><span class="s2">&#39;, the &#39;device_num_each_group&#39; must be greater than 1, &quot;</span>
                             <span class="sa">f</span><span class="s2">&quot;but got </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">group_device_num</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_check_data_dim</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="k">if</span> <span class="n">x</span><span class="o">.</span><span class="n">dim</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
            <span class="k">pass</span></div>


<div class="viewcode-block" id="SyncBatchNorm"><a class="viewcode-back" href="../../../../api_python/nn/mindspore.nn.SyncBatchNorm.html#mindspore.nn.SyncBatchNorm">[docs]</a><span class="k">class</span> <span class="nc">SyncBatchNorm</span><span class="p">(</span><span class="n">_BatchNorm</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Sync Batch Normalization layer over a N-dimension input.</span>

<span class="sd">    Sync Batch Normalization is cross device synchronized Batch Normalization. The implementation of Batch</span>
<span class="sd">    Normalization only normalizes the data within each device. Sync Batch Normalization will normalize the input</span>
<span class="sd">    within the group. It has been described in the paper `Batch Normalization: Accelerating Deep Network Training by</span>
<span class="sd">    Reducing Internal Covariate Shift &lt;https://arxiv.org/abs/1502.03167&gt;`_. It rescales and recenters the</span>
<span class="sd">    feature using a mini-batch of data and the learned parameters which can be described in the following formula.</span>

<span class="sd">    .. math::</span>
<span class="sd">        y = \frac{x - \mathrm{E}[x]}{\sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta</span>

<span class="sd">    Note:</span>
<span class="sd">        Currently, SyncBatchNorm only supports 2D and 4D inputs.</span>

<span class="sd">    Args:</span>
<span class="sd">        num_features (int): `C` from an expected input of size (N, C, H, W).</span>
<span class="sd">        eps (float): A value added to the denominator for numerical stability. Default: 1e-5.</span>
<span class="sd">        momentum (float): A floating hyperparameter of the momentum for the</span>
<span class="sd">            running_mean and running_var computation. Default: 0.9.</span>
<span class="sd">        affine (bool): A bool value. When set to True, gamma and beta can be learned. Default: True.</span>
<span class="sd">        gamma_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the gamma weight.</span>
<span class="sd">            The values of str refer to the function `initializer` including &#39;zeros&#39;, &#39;ones&#39;, &#39;xavier_uniform&#39;,</span>
<span class="sd">            &#39;he_uniform&#39;, etc. Default: &#39;ones&#39;.</span>
<span class="sd">        beta_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the beta weight.</span>
<span class="sd">            The values of str refer to the function `initializer` including &#39;zeros&#39;, &#39;ones&#39;, &#39;xavier_uniform&#39;,</span>
<span class="sd">            &#39;he_uniform&#39;, etc. Default: &#39;zeros&#39;.</span>
<span class="sd">        moving_mean_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the moving mean.</span>
<span class="sd">            The values of str refer to the function `initializer` including &#39;zeros&#39;, &#39;ones&#39;, &#39;xavier_uniform&#39;,</span>
<span class="sd">            &#39;he_uniform&#39;, etc. Default: &#39;zeros&#39;.</span>
<span class="sd">        moving_var_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the moving variance.</span>
<span class="sd">            The values of str refer to the function `initializer` including &#39;zeros&#39;, &#39;ones&#39;, &#39;xavier_uniform&#39;,</span>
<span class="sd">            &#39;he_uniform&#39;, etc. Default: &#39;ones&#39;.</span>
<span class="sd">        use_batch_statistics (bool): If true, use the mean value and variance value of current batch data. If false,</span>
<span class="sd">            use the mean value and variance value of specified value. If None, training process will use the mean and</span>
<span class="sd">            variance of current batch data and track the running mean and variance, eval process will use the running</span>
<span class="sd">            mean and variance. Default: None.</span>
<span class="sd">        process_groups (list): A list to divide devices into different sync groups, containing N subtraction lists.</span>
<span class="sd">            Each subtraction list contains int numbers identifying rank ids which need to be synchronized in the same</span>
<span class="sd">            group. All int values must be in [0, rank_size) and different from each other. Default: None, indicating</span>
<span class="sd">            synchronization across all devices.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **x** (Tensor) - Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})`.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, the normalized, scaled, offset tensor, of shape :math:`(N, C_{out}, H_{out}, W_{out})`.</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If `num_features` is not an int.</span>
<span class="sd">        TypeError: If `eps` is not a float.</span>
<span class="sd">        TypeError: If `process_groups` is not a list.</span>
<span class="sd">        ValueError: If `num_features` is less than 1.</span>
<span class="sd">        ValueError: If `momentum` is not in range [0, 1].</span>
<span class="sd">        ValueError: If rank_id in `process_groups` is not in range [0, rank_size).</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; # This example should be run with multiple processes.</span>
<span class="sd">        &gt;&gt;&gt; # Please refer to the tutorial &gt; Distributed Training on mindspore.cn.</span>
<span class="sd">        &gt;&gt;&gt; # Focus on the contents of these three parts: 配置分布式环境变量, 调用集合通信库, 运行脚本.</span>
<span class="sd">        &gt;&gt;&gt; import numpy as np</span>
<span class="sd">        &gt;&gt;&gt; from mindspore.communication import init</span>
<span class="sd">        &gt;&gt;&gt; from mindspore import context</span>
<span class="sd">        &gt;&gt;&gt; from mindspore.context import ParallelMode</span>
<span class="sd">        &gt;&gt;&gt; from mindspore import Tensor</span>
<span class="sd">        &gt;&gt;&gt; from mindspore import nn</span>
<span class="sd">        &gt;&gt;&gt; from mindspore import dtype as mstype</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; context.set_context(mode=context.GRAPH_MODE)</span>
<span class="sd">        &gt;&gt;&gt; init()</span>
<span class="sd">        &gt;&gt;&gt; context.reset_auto_parallel_context()</span>
<span class="sd">        &gt;&gt;&gt; context.set_auto_parallel_context(parallel_mode=ParallelMode.DATA_PARALLEL)</span>
<span class="sd">        &gt;&gt;&gt; sync_bn_op = nn.SyncBatchNorm(num_features=3, process_groups=[[0, 1], [2, 3]])</span>
<span class="sd">        &gt;&gt;&gt; x = Tensor(np.ones([1, 3, 2, 2]), mstype.float32)</span>
<span class="sd">        &gt;&gt;&gt; output = sync_bn_op(x)</span>
<span class="sd">        &gt;&gt;&gt; print(output)</span>
<span class="sd">        [[[[ 0.999995 0.999995 ]</span>
<span class="sd">           [ 0.999995 0.999995 ]]</span>
<span class="sd">          [[ 0.999995 0.999995 ]</span>
<span class="sd">           [ 0.999995 0.999995 ]]</span>
<span class="sd">          [[ 0.999995 0.999995 ]</span>
<span class="sd">           [ 0.999995 0.999995 ]]]]</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
                 <span class="n">num_features</span><span class="p">,</span>
                 <span class="n">eps</span><span class="o">=</span><span class="mf">1e-5</span><span class="p">,</span>
                 <span class="n">momentum</span><span class="o">=</span><span class="mf">0.9</span><span class="p">,</span>
                 <span class="n">affine</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
                 <span class="n">gamma_init</span><span class="o">=</span><span class="s1">&#39;ones&#39;</span><span class="p">,</span>
                 <span class="n">beta_init</span><span class="o">=</span><span class="s1">&#39;zeros&#39;</span><span class="p">,</span>
                 <span class="n">moving_mean_init</span><span class="o">=</span><span class="s1">&#39;zeros&#39;</span><span class="p">,</span>
                 <span class="n">moving_var_init</span><span class="o">=</span><span class="s1">&#39;ones&#39;</span><span class="p">,</span>
                 <span class="n">use_batch_statistics</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">process_groups</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize SyncBatchNorm.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">SyncBatchNorm</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_features</span><span class="p">,</span>
                                            <span class="n">eps</span><span class="p">,</span>
                                            <span class="n">momentum</span><span class="p">,</span>
                                            <span class="n">affine</span><span class="p">,</span>
                                            <span class="n">gamma_init</span><span class="p">,</span>
                                            <span class="n">beta_init</span><span class="p">,</span>
                                            <span class="n">moving_mean_init</span><span class="p">,</span>
                                            <span class="n">moving_var_init</span><span class="p">,</span>
                                            <span class="n">use_batch_statistics</span><span class="p">,</span>
                                            <span class="n">process_groups</span><span class="o">=</span><span class="n">process_groups</span><span class="p">,</span>
                                            <span class="n">input_dims</span><span class="o">=</span><span class="s1">&#39;both&#39;</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_check_data_dim</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="k">if</span> <span class="n">x</span><span class="o">.</span><span class="n">dim</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
            <span class="k">pass</span></div>


<span class="k">class</span> <span class="nc">LayerNorm</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Applies Layer Normalization over a mini-batch of inputs.</span>

<span class="sd">    Layer Normalization is widely used in recurrent neural networks. It applies</span>
<span class="sd">    normalization on a mini-batch of inputs for each single training case as described</span>
<span class="sd">    in the paper `Layer Normalization &lt;https://arxiv.org/pdf/1607.06450.pdf&gt;`_. Unlike Batch</span>
<span class="sd">    Normalization, Layer Normalization performs exactly the same computation at training and</span>
<span class="sd">    testing time. It can be described using the following formula. It is applied across all channels</span>
<span class="sd">    and pixel but only one batch size.</span>

<span class="sd">    .. math::</span>
<span class="sd">        y = \frac{x - \mathrm{E}[x]}{\sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta</span>

<span class="sd">    Args:</span>
<span class="sd">        normalized_shape (Union(tuple[int], list[int])): The normalization is performed over axis</span>
<span class="sd">            `begin_norm_axis ... R - 1`.</span>
<span class="sd">        begin_norm_axis (int): The first normalization dimension: normalization will be performed along dimensions</span>
<span class="sd">            `begin_norm_axis: rank(inputs)`, the value should be in [-1, rank(input)). Default: -1.</span>
<span class="sd">        begin_params_axis (int): The first parameter(beta, gamma)dimension: scale and centering parameters</span>
<span class="sd">            will have dimensions `begin_params_axis: rank(inputs)` and will be broadcast with</span>
<span class="sd">            the normalized inputs accordingly, the value should be in [-1, rank(input)). Default: -1.</span>
<span class="sd">        gamma_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the gamma weight.</span>
<span class="sd">            The values of str refer to the function `initializer` including &#39;zeros&#39;, &#39;ones&#39;, &#39;xavier_uniform&#39;,</span>
<span class="sd">            &#39;he_uniform&#39;, etc. Default: &#39;ones&#39;.</span>
<span class="sd">        beta_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the beta weight.</span>
<span class="sd">            The values of str refer to the function `initializer` including &#39;zeros&#39;, &#39;ones&#39;, &#39;xavier_uniform&#39;,</span>
<span class="sd">            &#39;he_uniform&#39;, etc. Default: &#39;zeros&#39;.</span>
<span class="sd">        epsilon (float): A value added to the denominator for numerical stability. Default: 1e-7.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **x** (Tensor) - The shape of &#39;x&#39; is :math:`(x_1, x_2, ..., x_R)`,</span>
<span class="sd">          and `input_shape[begin_norm_axis:]` is equal to `normalized_shape`.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, the normalized and scaled offset tensor, has the same shape and data type as the `x`.</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If `normalized_shape` is neither a list nor tuple.</span>
<span class="sd">        TypeError: If `begin_norm_axis` or `begin_params_axis` is not an int.</span>
<span class="sd">        TypeError: If `epsilon` is not a float.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU`` ``CPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; x = Tensor(np.ones([20, 5, 10, 10]), mindspore.float32)</span>
<span class="sd">        &gt;&gt;&gt; shape1 = x.shape[1:]</span>
<span class="sd">        &gt;&gt;&gt; m = nn.LayerNorm(shape1,  begin_norm_axis=1, begin_params_axis=1)</span>
<span class="sd">        &gt;&gt;&gt; output = m(x).shape</span>
<span class="sd">        &gt;&gt;&gt; print(output)</span>
<span class="sd">        (20, 5, 10, 10)</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
                 <span class="n">normalized_shape</span><span class="p">,</span>
                 <span class="n">begin_norm_axis</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span>
                 <span class="n">begin_params_axis</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span>
                 <span class="n">gamma_init</span><span class="o">=</span><span class="s1">&#39;ones&#39;</span><span class="p">,</span>
                 <span class="n">beta_init</span><span class="o">=</span><span class="s1">&#39;zeros&#39;</span><span class="p">,</span>
                 <span class="n">epsilon</span><span class="o">=</span><span class="mf">1e-7</span>
                 <span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize LayerNorm.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">LayerNorm</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">normalized_shape</span><span class="p">,</span> <span class="p">(</span><span class="nb">tuple</span><span class="p">,</span> <span class="nb">list</span><span class="p">)):</span>
            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="si">}</span><span class="s2">&#39;, the type of &#39;normalized_shape&#39; should be tuple[int] or list[int], &quot;</span>
                            <span class="sa">f</span><span class="s2">&quot;but got </span><span class="si">{</span><span class="n">normalized_shape</span><span class="si">}</span><span class="s2"> and the type is </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">normalized_shape</span><span class="p">)</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">normalized_shape</span> <span class="o">=</span> <span class="n">normalized_shape</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">begin_norm_axis</span> <span class="o">=</span> <span class="n">begin_norm_axis</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">begin_params_axis</span> <span class="o">=</span> <span class="n">begin_params_axis</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">epsilon</span> <span class="o">=</span> <span class="n">epsilon</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="n">Parameter</span><span class="p">(</span><span class="n">initializer</span><span class="p">(</span>
            <span class="n">gamma_init</span><span class="p">,</span> <span class="n">normalized_shape</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;gamma&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">=</span> <span class="n">Parameter</span><span class="p">(</span><span class="n">initializer</span><span class="p">(</span>
            <span class="n">beta_init</span><span class="p">,</span> <span class="n">normalized_shape</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;beta&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">layer_norm</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">LayerNorm</span><span class="p">(</span><span class="n">begin_norm_axis</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">begin_norm_axis</span><span class="p">,</span>
                                      <span class="n">begin_params_axis</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">begin_params_axis</span><span class="p">,</span>
                                      <span class="n">epsilon</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">epsilon</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">input_x</span><span class="p">):</span>
        <span class="n">y</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">layer_norm</span><span class="p">(</span><span class="n">input_x</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">beta</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">y</span>

    <span class="k">def</span> <span class="nf">extend_repr</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="s1">&#39;normalized_shape=</span><span class="si">{}</span><span class="s1">, begin_norm_axis=</span><span class="si">{}</span><span class="s1">, begin_params_axis=</span><span class="si">{}</span><span class="s1">, gamma</span><span class="si">{}</span><span class="s1">, beta=</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">normalized_shape</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">begin_norm_axis</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">begin_params_axis</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">beta</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">InstanceNorm2d</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Instance Normalization layer over a 4D input.</span>

<span class="sd">    This layer applies Instance Normalization over a 4D input (a mini-batch of 2D inputs with</span>
<span class="sd">    additional channel dimension) as described in the paper `Instance Normalization: The Missing Ingredient for</span>
<span class="sd">    Fast Stylization &lt;https://arxiv.org/abs/1607.08022&gt;`_. It rescales and recenters the feature using a mini-batch</span>
<span class="sd">    of data and the learned parameters which can be described in the following formula.</span>

<span class="sd">    .. math::</span>
<span class="sd">        y = \frac{x - \mathrm{E}[x]}{\sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta</span>

<span class="sd">    \gamma and \beta are learnable parameter vectors of size num_features if affine is True. The standard-deviation</span>
<span class="sd">    is calculated via the biased estimator.</span>

<span class="sd">    This layer uses instance statistics computed from input data in both training and evaluation modes.</span>

<span class="sd">    InstanceNorm2d and BatchNorm2d are very similar, but have some differences. InstanceNorm2d is applied on each</span>
<span class="sd">    channel of channeled data like RGB images, but BatchNorm2d is usually applied on each batch of batched data.</span>

<span class="sd">    Note:</span>
<span class="sd">        Note that the formula for updating the running_mean and running_var is</span>
<span class="sd">        :math:`\hat{x}_\text{new} = (1 - \text{momentum}) \times x_t + \text{momentum} \times \hat{x}`,</span>
<span class="sd">        where :math:`\hat{x}` is the estimated statistic and :math:`x_t` is the new observed value.</span>

<span class="sd">    Args:</span>
<span class="sd">        num_features (int): `C` from an expected input of size (N, C, H, W).</span>
<span class="sd">        eps (float): A value added to the denominator for numerical stability. Default: 1e-5.</span>
<span class="sd">        momentum (float): A floating hyperparameter of the momentum for the</span>
<span class="sd">            running_mean and running_var computation. Default: 0.1.</span>
<span class="sd">        affine (bool): A bool value. When set to True, gamma and beta can be learned. Default: True.</span>
<span class="sd">        gamma_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the gamma weight.</span>
<span class="sd">            The values of str refer to the function `initializer` including &#39;zeros&#39;, &#39;ones&#39;, etc. Default: &#39;ones&#39;.</span>
<span class="sd">        beta_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the beta weight.</span>
<span class="sd">            The values of str refer to the function `initializer` including &#39;zeros&#39;, &#39;ones&#39;, etc. Default: &#39;zeros&#39;.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **x** (Tensor) - Tensor of shape :math:`(N, C, H, W)`. Data type: float16 or float32.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, the normalized, scaled, offset tensor, of shape :math:`(N, C, H, W)`. Same type and</span>
<span class="sd">        shape as the `x`.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``GPU``</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If `num_features` is not an int.</span>
<span class="sd">        TypeError: If `eps` is not a float.</span>
<span class="sd">        TypeError: If `momentum` is not a float.</span>
<span class="sd">        TypeError: If `affine` is not a bool.</span>
<span class="sd">        TypeError: If the type of `gamma_init`/`beta_init` is not same, or if the initialized element type is not</span>
<span class="sd">            float32.</span>
<span class="sd">        ValueError: If `num_features` is less than 1.</span>
<span class="sd">        ValueError: If `momentum` is not in range [0, 1].</span>
<span class="sd">        KeyError: If any of `gamma_init`/`beta_init` is str and the homonymous class inheriting from `Initializer` not</span>
<span class="sd">            exists.</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; import mindspore</span>
<span class="sd">        &gt;&gt;&gt; import numpy as np</span>
<span class="sd">        &gt;&gt;&gt; import mindspore.nn as nn</span>
<span class="sd">        &gt;&gt;&gt; from mindspore import Tensor</span>
<span class="sd">        &gt;&gt;&gt; net = nn.InstanceNorm2d(3)</span>
<span class="sd">        &gt;&gt;&gt; x = Tensor(np.ones([2, 3, 2, 2]), mindspore.float32)</span>
<span class="sd">        &gt;&gt;&gt; output = net(x)</span>
<span class="sd">        &gt;&gt;&gt; print(output.shape)</span>
<span class="sd">        (2, 3, 2, 2)</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@cell_attr_register</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
                 <span class="n">num_features</span><span class="p">,</span>
                 <span class="n">eps</span><span class="o">=</span><span class="mf">1e-5</span><span class="p">,</span>
                 <span class="n">momentum</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
                 <span class="n">affine</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
                 <span class="n">gamma_init</span><span class="o">=</span><span class="s1">&#39;ones&#39;</span><span class="p">,</span>
                 <span class="n">beta_init</span><span class="o">=</span><span class="s1">&#39;zeros&#39;</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize InstanceNorm2d.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">InstanceNorm2d</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="n">validator</span><span class="o">.</span><span class="n">check_value_type</span><span class="p">(</span><span class="s1">&#39;num_features&#39;</span><span class="p">,</span> <span class="n">num_features</span><span class="p">,</span> <span class="p">[</span><span class="nb">int</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="p">)</span>
        <span class="n">validator</span><span class="o">.</span><span class="n">check_value_type</span><span class="p">(</span><span class="s1">&#39;eps&#39;</span><span class="p">,</span> <span class="n">eps</span><span class="p">,</span> <span class="p">[</span><span class="nb">float</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="p">)</span>
        <span class="n">validator</span><span class="o">.</span><span class="n">check_value_type</span><span class="p">(</span><span class="s1">&#39;momentum&#39;</span><span class="p">,</span> <span class="n">momentum</span><span class="p">,</span> <span class="p">[</span><span class="nb">float</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="p">)</span>
        <span class="n">validator</span><span class="o">.</span><span class="n">check_value_type</span><span class="p">(</span><span class="s1">&#39;affine&#39;</span><span class="p">,</span> <span class="n">affine</span><span class="p">,</span> <span class="p">[</span><span class="nb">bool</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="p">)</span>
        <span class="n">args_input</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;gamma_init&quot;</span><span class="p">:</span> <span class="n">gamma_init</span><span class="p">,</span> <span class="s2">&quot;beta_init&quot;</span><span class="p">:</span> <span class="n">beta_init</span><span class="p">}</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">check_types_valid</span><span class="p">(</span><span class="n">args_input</span><span class="p">,</span> <span class="s1">&#39;InstanceNorm2d&#39;</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">num_features</span> <span class="o">&lt;</span> <span class="mi">1</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="si">}</span><span class="s2">&#39;, the &#39;num_features&#39; must be at least 1, but got </span><span class="si">{</span><span class="n">num_features</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>

        <span class="k">if</span> <span class="n">momentum</span> <span class="o">&lt;</span> <span class="mi">0</span> <span class="ow">or</span> <span class="n">momentum</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="si">}</span><span class="s2">&#39;, the &#39;momentum&#39; should be a number in range [0, 1], &quot;</span>
                             <span class="sa">f</span><span class="s2">&quot;but got </span><span class="si">{</span><span class="n">momentum</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">num_features</span> <span class="o">=</span> <span class="n">num_features</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">eps</span> <span class="o">=</span> <span class="n">eps</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">input_dims</span> <span class="o">=</span> <span class="s1">&#39;2d&#39;</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">moving_mean</span> <span class="o">=</span> <span class="n">Parameter</span><span class="p">(</span><span class="n">initializer</span><span class="p">(</span><span class="s1">&#39;zeros&#39;</span><span class="p">,</span> <span class="n">num_features</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;mean&quot;</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">moving_variance</span> <span class="o">=</span> <span class="n">Parameter</span><span class="p">(</span><span class="n">initializer</span><span class="p">(</span><span class="s1">&#39;ones&#39;</span><span class="p">,</span> <span class="n">num_features</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;variance&quot;</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="n">Parameter</span><span class="p">(</span><span class="n">initializer</span><span class="p">(</span>
            <span class="n">gamma_init</span><span class="p">,</span> <span class="n">num_features</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;gamma&quot;</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="n">affine</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">=</span> <span class="n">Parameter</span><span class="p">(</span><span class="n">initializer</span><span class="p">(</span>
            <span class="n">beta_init</span><span class="p">,</span> <span class="n">num_features</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;beta&quot;</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="n">affine</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">shape</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Shape</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">momentum</span> <span class="o">=</span> <span class="n">momentum</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">instance_bn</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">InstanceNorm</span><span class="p">(</span><span class="n">epsilon</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">eps</span><span class="p">,</span> <span class="n">momentum</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">momentum</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_check_data_dim</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="k">raise</span> <span class="ne">NotImplementedError</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="n">_shape_check_bn</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">shape</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">input_dims</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="p">)</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">instance_bn</span><span class="p">(</span><span class="n">x</span><span class="p">,</span>
                                <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span><span class="p">,</span>
                                <span class="bp">self</span><span class="o">.</span><span class="n">beta</span><span class="p">,</span>
                                <span class="bp">self</span><span class="o">.</span><span class="n">moving_mean</span><span class="p">,</span>
                                <span class="bp">self</span><span class="o">.</span><span class="n">moving_variance</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>

    <span class="k">def</span> <span class="nf">extend_repr</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="s1">&#39;num_features=</span><span class="si">{}</span><span class="s1">, eps=</span><span class="si">{}</span><span class="s1">, momentum=</span><span class="si">{}</span><span class="s1">, gamma=</span><span class="si">{}</span><span class="s1">, beta=</span><span class="si">{}</span><span class="s1">, moving_mean=</span><span class="si">{}</span><span class="s1">, moving_variance=</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">num_features</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">eps</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">momentum</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">beta</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">moving_mean</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">moving_variance</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">check_types_valid</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">args_dict</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
        <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">_</span> <span class="ow">in</span> <span class="n">args_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
            <span class="n">val</span> <span class="o">=</span> <span class="n">args_dict</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
            <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">val</span><span class="p">,</span> <span class="p">(</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">numbers</span><span class="o">.</span><span class="n">Number</span><span class="p">,</span> <span class="nb">str</span><span class="p">,</span> <span class="n">Initializer</span><span class="p">)):</span>
                <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="si">}</span><span class="s2">&#39;, the type of &#39;</span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2">&#39; should be in &quot;</span>
                                <span class="sa">f</span><span class="s2">&quot;[Tensor, numbers.Number, str, Initializer], but got type </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">val</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>
            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">val</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">)</span> <span class="ow">and</span> <span class="n">val</span><span class="o">.</span><span class="n">dtype</span> <span class="o">!=</span> <span class="n">mstype</span><span class="o">.</span><span class="n">float32</span><span class="p">:</span>
                <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="si">}</span><span class="s2">&#39;, the type of &#39;</span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2">&#39; should be float32, &quot;</span>
                                <span class="sa">f</span><span class="s2">&quot;but got </span><span class="si">{</span><span class="n">val</span><span class="o">.</span><span class="n">dtype</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">GroupNorm</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Group Normalization over a mini-batch of inputs.</span>

<span class="sd">    Group Normalization is widely used in recurrent neural networks. It applies</span>
<span class="sd">    normalization on a mini-batch of inputs for each single training case as described</span>
<span class="sd">    in the paper `Group Normalization &lt;https://arxiv.org/pdf/1803.08494.pdf&gt;`_. Group Normalization</span>
<span class="sd">    divides the channels into groups and computes within each group the mean and variance for normalization,</span>
<span class="sd">    and it performs very stable over a wide range of batch size. It can be described using the following formula.</span>

<span class="sd">    .. math::</span>
<span class="sd">        y = \frac{x - \mathrm{E}[x]}{\sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta</span>

<span class="sd">    Args:</span>
<span class="sd">        num_groups (int): The number of groups to be divided along the channel dimension.</span>
<span class="sd">        num_channels (int): The number of channels per group.</span>
<span class="sd">        eps (float): A value added to the denominator for numerical stability. Default: 1e-5.</span>
<span class="sd">        affine (bool): A bool value, this layer will have learnable affine parameters when set to true. Default: True.</span>
<span class="sd">        gamma_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the gamma weight.</span>
<span class="sd">            The values of str refer to the function `initializer` including &#39;zeros&#39;, &#39;ones&#39;, &#39;xavier_uniform&#39;,</span>
<span class="sd">            &#39;he_uniform&#39;, etc. Default: &#39;ones&#39;. If gamma_init is a Tensor, the shape must be [num_channels].</span>
<span class="sd">        beta_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the beta weight.</span>
<span class="sd">            The values of str refer to the function `initializer` including &#39;zeros&#39;, &#39;ones&#39;, &#39;xavier_uniform&#39;,</span>
<span class="sd">            &#39;he_uniform&#39;, etc. Default: &#39;zeros&#39;. If beta_init is a Tensor, the shape must be [num_channels].</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **x** (Tensor) - The input feature with shape [N, C, H, W].</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, the normalized and scaled offset tensor, has the same shape and data type as the `x`.</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If `num_groups` or `num_channels` is not an int.</span>
<span class="sd">        TypeError: If `eps` is not a float.</span>
<span class="sd">        TypeError: If `affine` is not a bool.</span>
<span class="sd">        ValueError: If `num_groups` or `num_channels` is less than 1.</span>
<span class="sd">        ValueError: If `num_channels` is not divided by `num_groups`.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU`` ``CPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; group_norm_op = nn.GroupNorm(2, 2)</span>
<span class="sd">        &gt;&gt;&gt; x = Tensor(np.ones([1, 2, 4, 4], np.float32))</span>
<span class="sd">        &gt;&gt;&gt; output = group_norm_op(x)</span>
<span class="sd">        &gt;&gt;&gt; print(output)</span>
<span class="sd">        [[[[0. 0. 0. 0.]</span>
<span class="sd">           [0. 0. 0. 0.]</span>
<span class="sd">           [0. 0. 0. 0.]</span>
<span class="sd">           [0. 0. 0. 0.]]</span>
<span class="sd">          [[0. 0. 0. 0.]</span>
<span class="sd">           [0. 0. 0. 0.]</span>
<span class="sd">           [0. 0. 0. 0.]</span>
<span class="sd">           [0. 0. 0. 0.]]]]</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">num_groups</span><span class="p">,</span> <span class="n">num_channels</span><span class="p">,</span> <span class="n">eps</span><span class="o">=</span><span class="mf">1e-05</span><span class="p">,</span> <span class="n">affine</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">gamma_init</span><span class="o">=</span><span class="s1">&#39;ones&#39;</span><span class="p">,</span> <span class="n">beta_init</span><span class="o">=</span><span class="s1">&#39;zeros&#39;</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Initialize GroupNorm.&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">GroupNorm</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">num_groups</span> <span class="o">=</span> <span class="n">validator</span><span class="o">.</span><span class="n">check_positive_int</span><span class="p">(</span><span class="n">num_groups</span><span class="p">,</span> <span class="s2">&quot;num_groups&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">num_channels</span> <span class="o">=</span> <span class="n">validator</span><span class="o">.</span><span class="n">check_positive_int</span><span class="p">(</span><span class="n">num_channels</span><span class="p">,</span> <span class="s2">&quot;num_channels&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">num_channels</span> <span class="o">%</span> <span class="n">num_groups</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;For &#39;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="si">}</span><span class="s2">&#39;, the &#39;num_channels&#39; should be divided by &#39;num_groups&#39;, &quot;</span>
                             <span class="sa">f</span><span class="s2">&quot;but got &#39;num_channels&#39;: </span><span class="si">{</span><span class="n">num_channels</span><span class="si">}</span><span class="s2">, &#39;num_groups&#39;: </span><span class="si">{</span><span class="n">num_groups</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">eps</span> <span class="o">=</span> <span class="n">validator</span><span class="o">.</span><span class="n">check_value_type</span><span class="p">(</span><span class="s1">&#39;eps&#39;</span><span class="p">,</span> <span class="n">eps</span><span class="p">,</span> <span class="p">(</span><span class="nb">float</span><span class="p">,),</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">affine</span> <span class="o">=</span> <span class="n">validator</span><span class="o">.</span><span class="n">check_bool</span><span class="p">(</span><span class="n">affine</span><span class="p">,</span> <span class="n">arg_name</span><span class="o">=</span><span class="s2">&quot;affine&quot;</span><span class="p">,</span> <span class="n">prim_name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="n">Parameter</span><span class="p">(</span><span class="n">initializer</span><span class="p">(</span>
            <span class="n">gamma_init</span><span class="p">,</span> <span class="n">num_channels</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;gamma&quot;</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="n">affine</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">=</span> <span class="n">Parameter</span><span class="p">(</span><span class="n">initializer</span><span class="p">(</span>
            <span class="n">beta_init</span><span class="p">,</span> <span class="n">num_channels</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;beta&quot;</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="n">affine</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">shape</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">shape</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">reshape</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">reshape</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">reduce_mean</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">ReduceMean</span><span class="p">(</span><span class="n">keep_dims</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">square</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">square</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">reduce_sum</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">ReduceSum</span><span class="p">(</span><span class="n">keep_dims</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">sqrt</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Sqrt</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">_cal_output</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;calculate groupnorm output&quot;&quot;&quot;</span>
        <span class="n">batch</span><span class="p">,</span> <span class="n">channel</span><span class="p">,</span> <span class="n">height</span><span class="p">,</span> <span class="n">width</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">shape</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
        <span class="n">_channel_check</span><span class="p">(</span><span class="n">channel</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_channels</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="p">)</span>
        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="p">(</span><span class="n">batch</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_groups</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">))</span>
        <span class="n">mean</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">reduce_mean</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
        <span class="n">var</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">reduce_sum</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">square</span><span class="p">(</span><span class="n">x</span> <span class="o">-</span> <span class="n">mean</span><span class="p">),</span> <span class="mi">2</span><span class="p">)</span> <span class="o">/</span> <span class="p">(</span><span class="n">channel</span> <span class="o">*</span> <span class="n">height</span> <span class="o">*</span> <span class="n">width</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_groups</span><span class="p">)</span>
        <span class="n">std</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">var</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">eps</span><span class="p">)</span>
        <span class="n">x</span> <span class="o">=</span> <span class="p">(</span><span class="n">x</span> <span class="o">-</span> <span class="n">mean</span><span class="p">)</span> <span class="o">/</span> <span class="n">std</span>
        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="p">(</span><span class="n">batch</span><span class="p">,</span> <span class="n">channel</span><span class="p">,</span> <span class="n">height</span><span class="p">,</span> <span class="n">width</span><span class="p">))</span>
        <span class="n">output</span> <span class="o">=</span> <span class="n">x</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">gamma</span><span class="p">,</span> <span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">beta</span><span class="p">,</span> <span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span>
        <span class="k">return</span> <span class="n">output</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="n">_shape_check</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">shape</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="p">)</span>
        <span class="n">output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_cal_output</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">output</span>

    <span class="k">def</span> <span class="nf">extend_repr</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="s1">&#39;num_groups=</span><span class="si">{}</span><span class="s1">, num_channels=</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">num_groups</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_channels</span><span class="p">)</span>
</pre></div>

           </div>
           
          </div>
          <footer>

  <hr/>

  <div role="contentinfo">
    <p>
        &#169; Copyright 2021, MindSpore.

    </p>
  </div>
    
    
    
    Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
    
    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
    
    provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>
        </div>
      </div>

    </section>

  </div>
  

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
   

</body>
</html>